import json
from pymongo import MongoClient as Client
from my_fake_useragent import UserAgent
import requests
from bs4 import BeautifulSoup as bs
import time
import threading
ua = UserAgent()
header = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
                        'AppleWebKit/537.36 (KHTML, like Gecko) '
                        'Chrome/64.0.3282.186 Safari/537.36'}
def dictProxy(dic):
    s = dic['type'] + '://' + dic['ip'] + ':' +str(dic['port'])
    return {'http': s, 'https': s}

def write_mongo(ip_pool):
    client = Client(host='localhost', port=27017)
    db = client['proxy_db']
    coll = db['proxies']
    for ip in ip_pool:
        if coll.find({'ip':ip['ip']}).count() == 0:
            coll.insert_one(ip)
    client.close()

def write_json(ip_pool):
    json_str = json.dumps(ip_pool, indent=4)
    with open('proxy.json','w', encoding='utf-8') as f:
        f.write(json_str)

def parse_item(items):
    ips = []
    for item in items:
        tds = item.find_all('td')
        ip, port, _type = tds[1].text, int(tds[2].text), tds[5].text.lower()
        ips.append({
            'ip': ip,
            'port': port,
            'type': _type
        })
    return ips

def check_ip(ip):
    try:
        pro = dictProxy(ip)
        url = 'https://www.ipip.net/'
        r = requests.get(url, headers=header, proxies=pro, timeout=5)
        r.raise_for_status()
        print(r.status_code,ip['ip'])
        return True
    except Exception as e:
        print(e)
        pass



def get_proxies(index):
    url = 'http://www.xicidaili.com/nt/%d' % index
    r = requests.get(url, headers=header)
    r.encoding = r.apparent_encoding
    r.raise_for_status()
    soup = bs(r.text, 'lxml')
    # 第一个是显示最上方的信息的，需要丢掉
    items = soup.find_all('tr')[1:]
    ips = parse_item(items)
    good_proxies = []
    for ip in ips:
        if check_ip(ip): # 如果要检查ip可用性，解除该注释
            good_proxies.append(ip)
    return good_proxies


if __name__ == '__main__':
    proxies = get_proxies(100) # 页数，任意输
    write_json(proxies)
    # write_mongo(proxies)



